/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.protocol.ftp;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.util.List;
//import java.util.LinkedList;

import org.apache.commons.net.MalformedServerReplyException;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPCommand;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileEntryParser;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.net.ftp.FTPConnectionClosedException;

/***********************************************
 * Client.java encapsulates functionalities necessary for nutch to get dir list
 * and retrieve file from an FTP server. This class takes care of all low level
 * details of interacting with an FTP server and provides a convenient higher
 * level interface.
 * 
 * Modified from FtpClient.java in apache commons-net.
 * 
 * Notes by John Xing: ftp server implementations are hardly uniform and none
 * seems to follow RFCs whole-heartedly. We have no choice, but assume common
 * denominator as following: (1) Use stream mode for data transfer. Block mode
 * will be better for multiple file downloading and partial file downloading.
 * However not every ftpd has block mode support. (2) Use passive mode for data
 * connection. So nutch will work if we run behind firewall. (3) Data connection
 * is opened/closed per ftp command for the reasons listed in (1). There are ftp
 * servers out there, when partial downloading is enforeced by closing data
 * channel socket on our client side, the server side immediately closes control
 * channel (socket). Our codes deal with such a bad behavior. (4) LIST is used
 * to obtain remote file attributes if possible. MDTM & SIZE would be nice, but
 * not as ubiquitously implemented as LIST. (5) Avoid using ABOR in single
 * thread? Do not use it at all.
 * 
 * About exceptions: Some specific exceptions are re-thrown as one of
 * FtpException*.java In fact, each function throws FtpException*.java or pass
 * IOException.
 * 
 * @author John Xing
 ***********************************************/

public class Client extends FTP {
	private int __dataTimeout;
	private int __passivePort;
	private String __passiveHost;
	// private int __fileType, __fileFormat;
	private boolean __remoteVerificationEnabled;
	// private FTPFileEntryParser __entryParser;
	private String __systemName;

	// constructor
	public Client() {
		__initDefaults();
		__dataTimeout = -1;
		__remoteVerificationEnabled = true;
	}

	// defaults when initialize
	private void __initDefaults() {
		__passiveHost = null;
		__passivePort = -1;
		__systemName = null;
		/*
		 * __fileType = FTP.ASCII_FILE_TYPE; __fileFormat =
		 * FTP.NON_PRINT_TEXT_FORMAT; __entryParser = null;
		 */}

	// parse reply for pass()
	private void __parsePassiveModeReply(String reply)
			throws MalformedServerReplyException {
		int i, index, lastIndex;
		String octet1, octet2;
		StringBuffer host;

		reply = reply.substring(reply.indexOf('(') + 1, reply.indexOf(')'))
				.trim();

		host = new StringBuffer(24);
		lastIndex = 0;
		index = reply.indexOf(',');
		host.append(reply.substring(lastIndex, index));

		for (i = 0; i < 3; i++) {
			host.append('.');
			lastIndex = index + 1;
			index = reply.indexOf(',', lastIndex);
			host.append(reply.substring(lastIndex, index));
		}

		lastIndex = index + 1;
		index = reply.indexOf(',', lastIndex);

		octet1 = reply.substring(lastIndex, index);
		octet2 = reply.substring(index + 1);

		// index and lastIndex now used as temporaries
		try {
			index = Integer.parseInt(octet1);
			lastIndex = Integer.parseInt(octet2);
		} catch (NumberFormatException e) {
			throw new MalformedServerReplyException(
					"Could not parse passive host information.\nServer Reply: "
							+ reply);
		}

		index <<= 8;
		index |= lastIndex;

		__passiveHost = host.toString();
		__passivePort = index;
	}

	/**
	 * open a passive data connection socket
	 * 
	 * @param command
	 * @param arg
	 * @return
	 * @throws IOException
	 * @throws FtpExceptionCanNotHaveDataConnection
	 */
	protected Socket __openPassiveDataConnection(int command, String arg)
			throws IOException, FtpExceptionCanNotHaveDataConnection {
		Socket socket;

		// // 20040317, xing, accommodate ill-behaved servers, see below
		// int port_previous = __passivePort;

		if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
			throw new FtpExceptionCanNotHaveDataConnection(
					"pasv() failed. " + getReplyString());

		try {
			__parsePassiveModeReply(getReplyStrings()[0]);
		} catch (MalformedServerReplyException e) {
			throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
		}

		// // 20040317, xing, accommodate ill-behaved servers, see above
		// int count = 0;
		// System.err.println("__passivePort "+__passivePort);
		// System.err.println("port_previous "+port_previous);
		// while (__passivePort == port_previous) {
		// // just quit if too many tries. make it an exception here?
		// if (count++ > 10)
		// return null;
		// // slow down further for each new try
		// Thread.sleep(500*count);
		// if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
		// throw new FtpExceptionCanNotHaveDataConnection(
		// "pasv() failed. " + getReplyString());
		// //return null;
		// try {
		// __parsePassiveModeReply(getReplyStrings()[0]);
		// } catch (MalformedServerReplyException e) {
		// throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
		// }
		// }

		socket = _socketFactory_.createSocket(__passiveHost, __passivePort);

		if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) {
			socket.close();
			return null;
		}

		if (__remoteVerificationEnabled && !verifyRemote(socket)) {
			InetAddress host1, host2;

			host1 = socket.getInetAddress();
			host2 = getRemoteAddress();

			socket.close();

			// our precaution
			throw new FtpExceptionCanNotHaveDataConnection(
					"Host attempting data connection " + host1.getHostAddress()
							+ " is not same as server " + host2.getHostAddress()
							+ " So we intentionally close it for security precaution.");
		}

		if (__dataTimeout >= 0)
			socket.setSoTimeout(__dataTimeout);

		return socket;
	}

	/***
	 * Sets the timeout in milliseconds to use for data connection. set
	 * immediately after opening the data connection.
	 ***/
	public void setDataTimeout(int timeout) {
		__dataTimeout = timeout;
	}

	/***
	 * Closes the connection to the FTP server and restores connection
	 * parameters to the default values.
	 * <p>
	 * 
	 * @exception IOException
	 *                If an error occurs while disconnecting.
	 ***/
	public void disconnect() throws IOException {
		__initDefaults();
		super.disconnect();
		// no worry for data connection, since we always close it
		// in every ftp command that invloves data connection
	}

	/***
	 * Enable or disable verification that the remote host taking part of a data
	 * connection is the same as the host to which the control connection is
	 * attached. The default is for verification to be enabled. You may set this
	 * value at any time, whether the FTPClient is currently connected or not.
	 * <p>
	 * 
	 * @param enable
	 *            True to enable verification, false to disable verification.
	 ***/
	public void setRemoteVerificationEnabled(boolean enable) {
		__remoteVerificationEnabled = enable;
	}

	/***
	 * Return whether or not verification of the remote host participating in
	 * data connections is enabled. The default behavior is for verification to
	 * be enabled.
	 * <p>
	 * 
	 * @return True if verification is enabled, false if not.
	 ***/
	public boolean isRemoteVerificationEnabled() {
		return __remoteVerificationEnabled;
	}

	/***
	 * Login to the FTP server using the provided username and password.
	 * <p>
	 * 
	 * @param username
	 *            The username to login under.
	 * @param password
	 *            The password to use.
	 * @return True if successfully completed, false if not.
	 * @exception FTPConnectionClosedException
	 *                If the FTP server prematurely closes the connection as a
	 *                result of the client being idle or some other reason
	 *                causing the server to send FTP reply code 421. This
	 *                exception may be caught either as an IOException or
	 *                independently as itself.
	 * @exception IOException
	 *                If an I/O error occurs while either sending a command to
	 *                the server or receiving a reply from the server.
	 ***/
	public boolean login(String username, String password) throws IOException {
		user(username);

		if (FTPReply.isPositiveCompletion(getReplyCode()))
			return true;

		// If we get here, we either have an error code, or an intermmediate
		// reply requesting password.
		if (!FTPReply.isPositiveIntermediate(getReplyCode()))
			return false;

		return FTPReply.isPositiveCompletion(pass(password));
	}

	/***
	 * Logout of the FTP server by sending the QUIT command.
	 * <p>
	 * 
	 * @return True if successfully completed, false if not.
	 * @exception FTPConnectionClosedException
	 *                If the FTP server prematurely closes the connection as a
	 *                result of the client being idle or some other reason
	 *                causing the server to send FTP reply code 421. This
	 *                exception may be caught either as an IOException or
	 *                independently as itself.
	 * @exception IOException
	 *                If an I/O error occurs while either sending a command to
	 *                the server or receiving a reply from the server.
	 ***/
	public boolean logout() throws IOException {
		return FTPReply.isPositiveCompletion(quit());
	}

	/**
	 * Retrieve a list reply for path
	 * 
	 * @param path
	 * @param entries
	 * @param limit
	 * @param parser
	 * @throws IOException
	 * @throws FtpExceptionCanNotHaveDataConnection
	 * @throws FtpExceptionUnknownForcedDataClose
	 * @throws FtpExceptionControlClosedByForcedDataClose
	 */
	public void retrieveList(String path, List<FTPFile> entries, int limit,
			FTPFileEntryParser parser)
			throws IOException, FtpExceptionCanNotHaveDataConnection,
			FtpExceptionUnknownForcedDataClose,
			FtpExceptionControlClosedByForcedDataClose {
		Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);

		if (socket == null)
			throw new FtpExceptionCanNotHaveDataConnection(
					"LIST " + ((path == null) ? "" : path));

		BufferedReader reader = new BufferedReader(
				new InputStreamReader(socket.getInputStream()));

		// force-close data channel socket, when download limit is reached
		// boolean mandatory_close = false;

		// List entries = new LinkedList();
		int count = 0;
		String line = parser.readNextEntry(reader);
		while (line != null) {
			FTPFile ftpFile = parser.parseFTPEntry(line);
			// skip non-formatted lines
			if (ftpFile == null) {
				line = parser.readNextEntry(reader);
				continue;
			}
			entries.add(ftpFile);
			count += line.length();
			// impose download limit if limit >= 0, otherwise no limit
			// here, cut off is up to the line when total bytes is just over
			// limit
			if (limit >= 0 && count > limit) {
				// mandatory_close = true;
				break;
			}
			line = parser.readNextEntry(reader);
		}

		// if (mandatory_close)
		// you always close here, no matter mandatory_close or not.
		// however different ftp servers respond differently, see below.
		socket.close();

		// scenarios:
		// (1) mandatory_close is false, download limit not reached
		// no special care here
		// (2) mandatory_close is true, download limit is reached
		// different servers have different reply codes:

		try {
			int reply = getReply();
			if (!_notBadReply(reply))
				throw new FtpExceptionUnknownForcedDataClose(getReplyString());
		} catch (FTPConnectionClosedException e) {
			// some ftp servers will close control channel if data channel
			// socket
			// is closed by our end before all data has been read out. Check:
			// tux414.q-tam.hp.com FTP server (hp.com version whp02)
			// so must catch FTPConnectionClosedException thrown by getReply()
			// above
			// disconnect();
			throw new FtpExceptionControlClosedByForcedDataClose(
					e.getMessage());
		}

	}

	/**
	 * Retrieve a file for path
	 * 
	 * @param path
	 * @param os
	 * @param limit
	 * @throws IOException
	 * @throws FtpExceptionCanNotHaveDataConnection
	 * @throws FtpExceptionUnknownForcedDataClose
	 * @throws FtpExceptionControlClosedByForcedDataClose
	 */
	public void retrieveFile(String path, OutputStream os, int limit)
			throws IOException, FtpExceptionCanNotHaveDataConnection,
			FtpExceptionUnknownForcedDataClose,
			FtpExceptionControlClosedByForcedDataClose {

		Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);

		if (socket == null)
			throw new FtpExceptionCanNotHaveDataConnection(
					"RETR " + ((path == null) ? "" : path));

		InputStream input = socket.getInputStream();

		// 20040318, xing, treat everything as BINARY_FILE_TYPE for now
		// do we ever need ASCII_FILE_TYPE?
		// if (__fileType == ASCII_FILE_TYPE)
		// input = new FromNetASCIIInputStream(input);

		// fixme, should we instruct server here for binary file type?

		// force-close data channel socket
		// boolean mandatory_close = false;

		int len;
		int count = 0;
		byte[] buf = new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE];
		while ((len = input.read(buf, 0, buf.length)) != -1) {
			count += len;
			// impose download limit if limit >= 0, otherwise no limit
			// here, cut off is exactly of limit bytes
			if (limit >= 0 && count > limit) {
				os.write(buf, 0, len - (count - limit));
				// mandatory_close = true;
				break;
			}
			os.write(buf, 0, len);
			os.flush();
		}

		// if (mandatory_close)
		// you always close here, no matter mandatory_close or not.
		// however different ftp servers respond differently, see below.
		socket.close();

		// scenarios:
		// (1) mandatory_close is false, download limit not reached
		// no special care here
		// (2) mandatory_close is true, download limit is reached
		// different servers have different reply codes:

		// do not need this
		// sendCommand("ABOR");

		try {
			int reply = getReply();
			if (!_notBadReply(reply))
				throw new FtpExceptionUnknownForcedDataClose(getReplyString());
		} catch (FTPConnectionClosedException e) {
			// some ftp servers will close control channel if data channel
			// socket
			// is closed by our end before all data has been read out. Check:
			// tux414.q-tam.hp.com FTP server (hp.com version whp02)
			// so must catch FTPConnectionClosedException thrown by getReply()
			// above
			// disconnect();
			throw new FtpExceptionControlClosedByForcedDataClose(
					e.getMessage());
		}

	}

	// reply check after closing data connection
	private boolean _notBadReply(int reply) {

		if (FTPReply.isPositiveCompletion(reply)) {
			// do nothing
		} else if (reply == 426) { // FTPReply.TRANSFER_ABORTED
			// some ftp servers reply 426, e.g.,
			// foggy FTP server (Version wu-2.6.2(2)
			// there is second reply witing? no!
			// getReply();
		} else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN
			// some ftp servers reply 450, e.g.,
			// ProFTPD [ftp.kernel.org]
			// there is second reply witing? no!
			// getReply();
		} else if (reply == 451) { // FTPReply.ACTION_ABORTED
			// some ftp servers reply 451, e.g.,
			// ProFTPD [ftp.kernel.org]
			// there is second reply witing? no!
			// getReply();
		} else if (reply == 451) { // FTPReply.ACTION_ABORTED
		} else {
			// what other kind of ftp server out there?
			return false;
		}

		return true;
	}

	/***
	 * Sets the file type to be transferred. This should be one of
	 * <code> FTP.ASCII_FILE_TYPE </code>, <code> FTP.IMAGE_FILE_TYPE </code>,
	 * etc. The file type only needs to be set when you want to change the type.
	 * After changing it, the new type stays in effect until you change it
	 * again. The default file type is <code> FTP.ASCII_FILE_TYPE </code> if
	 * this method is never called.
	 * <p>
	 * 
	 * @param fileType
	 *            The <code> _FILE_TYPE </code> constant indcating the type of
	 *            file.
	 * @return True if successfully completed, false if not.
	 * @exception FTPConnectionClosedException
	 *                If the FTP server prematurely closes the connection as a
	 *                result of the client being idle or some other reason
	 *                causing the server to send FTP reply code 421. This
	 *                exception may be caught either as an IOException or
	 *                independently as itself.
	 * @exception IOException
	 *                If an I/O error occurs while either sending a command to
	 *                the server or receiving a reply from the server.
	 ***/
	public boolean setFileType(int fileType) throws IOException {
		if (FTPReply.isPositiveCompletion(type(fileType))) {
			/*
			 * __fileType = fileType; __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
			 */
			return true;
		}
		return false;
	}

	/***
	 * Fetches the system type name from the server and returns the string. This
	 * value is cached for the duration of the connection after the first call
	 * to this method. In other words, only the first time that you invoke this
	 * method will it issue a SYST command to the FTP server. FTPClient will
	 * remember the value and return the cached value until a call to
	 * disconnect.
	 * <p>
	 * 
	 * @return The system type name obtained from the server. null if the
	 *         information could not be obtained.
	 * @exception FTPConnectionClosedException
	 *                If the FTP server prematurely closes the connection as a
	 *                result of the client being idle or some other reason
	 *                causing the server to send FTP reply code 421. This
	 *                exception may be caught either as an IOException or
	 *                independently as itself.
	 * @exception IOException
	 *                If an I/O error occurs while either sending a command to
	 *                the server or receiving a reply from the server.
	 ***/
	public String getSystemName()
			throws IOException, FtpExceptionBadSystResponse {
		// if (syst() == FTPReply.NAME_SYSTEM_TYPE)
		// Technically, we should expect a NAME_SYSTEM_TYPE response, but
		// in practice FTP servers deviate, so we soften the condition to
		// a positive completion.
		if (__systemName == null && FTPReply.isPositiveCompletion(syst())) {
			__systemName = (getReplyStrings()[0]).substring(4);
		} else {
			throw new FtpExceptionBadSystResponse(
					"Bad response of SYST: " + getReplyString());
		}

		return __systemName;
	}

	/***
	 * Sends a NOOP command to the FTP server. This is useful for preventing
	 * server timeouts.
	 * <p>
	 * 
	 * @return True if successfully completed, false if not.
	 * @exception FTPConnectionClosedException
	 *                If the FTP server prematurely closes the connection as a
	 *                result of the client being idle or some other reason
	 *                causing the server to send FTP reply code 421. This
	 *                exception may be caught either as an IOException or
	 *                independently as itself.
	 * @exception IOException
	 *                If an I/O error occurs while either sending a command to
	 *                the server or receiving a reply from the server.
	 ***/
	public boolean sendNoOp() throws IOException {
		return FTPReply.isPositiveCompletion(noop());
	}

	// client.stat(path);
	// client.sendCommand("STAT");
	// client.sendCommand("STAT",path);
	// client.sendCommand("MDTM",path);
	// client.sendCommand("SIZE",path);
	// client.sendCommand("HELP","SITE");
	// client.sendCommand("SYST");
	// client.setRestartOffset(120);

}
